{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "74e3a0bf",
   "metadata": {},
   "source": [
    "## DOWNLOAD HST PRODUCTS VIA PROPOSAL ID\n",
    "\n",
    "This notebook consists of 3 cells\n",
    "1. imports, define main working directory, main Downloader function that queries MAST for observations\n",
    "2. Read in csv file of targets, build list of targets to loop over\n",
    "3. Loop over list of targets and download data\n",
    "\n",
    "last ran 2025.10.27 by C. Watson "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "bfd27475",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 385
    },
    "executionInfo": {
     "elapsed": 2072,
     "status": "error",
     "timestamp": 1750445557272,
     "user": {
      "displayName": "Gandab Mammadova",
      "userId": "00032345103725394396"
     },
     "user_tz": 240
    },
    "id": "bfd27475",
    "outputId": "c994075e-9f7d-491f-9a03-34a350e7a7cd"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import glob\n",
    "import os\n",
    "import shutil\n",
    "from astroquery.mast import Observations\n",
    "\n",
    "# Directory where the data will be stored\n",
    "Main_dir = '/Users/cwat/research/agel_hst/HST'  # Change this to your main data directory\n",
    "\n",
    "\n",
    "def Downloader(lens_name, target_name, band, camera, proposal_id_no):\n",
    "    \"\"\"\n",
    "    Lens Name : User Intended Name for the Lenses (For the folder names)\n",
    "    Target Name : Target Name of the Lens in the MAST Archive\n",
    "    Band: Filter band of the HST Image\n",
    "    Camera: Camera of the HST IMage\n",
    "    Proposal ID No : Proposal ID of the Image\n",
    "    \"\"\"\n",
    "\n",
    "    download_dir = os.path.join(Main_dir, f'{lens_name}/HST/{proposal_id_no}_{band}/raw_data/') #Gives the download directory\n",
    "\n",
    "    obs_table = Observations.query_criteria(proposal_id=proposal_id_no, target_name=target_name,\n",
    "                                        filters=band)\n",
    "\n",
    "    print(obs_table)\n",
    "\n",
    "    if band == \"F140W\":\n",
    "        file_format=['FLT']  # Different file format for IR images\n",
    "    else:\n",
    "        file_format=['FLC']\n",
    "\n",
    "    # Download the data\n",
    "    download_tab = Observations.download_products(obs_table['obsid'], mrp_only=False,\n",
    "                                              download_dir=download_dir,\n",
    "                                              productSubGroupDescription=file_format)\n",
    "\n",
    "    # Move the science files to the main directory and clean up\n",
    "    science_files = glob.glob(os.path.join(os.curdir, download_dir, 'mastDownload', 'HST', '*', '*fits'))\n",
    "    for im in science_files:\n",
    "        shutil.copy(im, download_dir)\n",
    "    # Remove mastDownload folder\n",
    "    shutil.rmtree(download_dir + 'mastDownload')\n",
    "    # Remove HST files\n",
    "    fileList = glob.glob(download_dir+'/hst*')\n",
    "\n",
    "    for filePath in fileList:   # Iterate over the list of filepaths & remove each file.\n",
    "        try:\n",
    "            os.remove(filePath)\n",
    "        except:\n",
    "            print(\"Error while deleting file : \", filePath)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "0eefe42a",
   "metadata": {
    "id": "0eefe42a"
   },
   "outputs": [],
   "source": [
    "# Glazerbrook proposal ID 16773 uses 2 filters on WFC3, Huang uses just F140W, Tran uses F606W on ACS\n",
    "proposal_id_no = '17307'\n",
    "filters = ['F606W']#, 'F200LP'] \n",
    "camera = 'ACS'\n",
    "\n",
    "# csv with data info \n",
    "#       -- took this one from Airtable Parent Catalog filtered \n",
    "#              to targets with Glazebrook HST proposal tag -- had to manually change some target name to match what is in observation header\n",
    "#       -- mostly want to make sure you have the target name and the AGEL name (if different)\n",
    "## Note: you need to make sure that the target names in this csv match exactly what is in MAST, otherwise the query will return nothing. These can be different from the AGEL names and different from the names you use to name your directories\n",
    "Lens_List = pd.read_csv(f'{Main_dir}/{proposal_id_no}_targets.csv') \n",
    "\n",
    "Processed_Lenses=[]\n",
    "for index, row in Lens_List.iterrows():\n",
    "    Processed_Lenses.append([row['objname'],row['catalogue_objname']])\n",
    "                              # AGEL name      MAST target name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4d65d4f3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "intentType obs_collection provenance_name ... srcDen   obsid     objID   \n",
      "---------- -------------- --------------- ... ------ --------- ----------\n",
      "   science            HST          CALACS ...    nan 368595406 1062444581\n",
      "   science            HST         HAP-SVM ...    nan 368792258 1063139279\n",
      "INFO: 4 of 8 products were duplicates. Only returning 4 unique product(s). [astroquery.mast.utils]\n",
      "Downloading URL https://mast.stsci.edu/api/v0.1/Download/file?uri=mast:HST/product/hst_17307_53_acs_wfc_f606w_jf5453c0_flc.fits to /Users/cwat/research/agel_hst/HST/AGEL051603-220847A/HST/17307_F606W/raw_data/mastDownload/HST/hst_17307_53_acs_wfc_f606w_jf5453c0/hst_17307_53_acs_wfc_f606w_jf5453c0_flc.fits ... [Done]\n",
      "Downloading URL https://mast.stsci.edu/api/v0.1/Download/file?uri=mast:HST/product/hst_17307_53_acs_wfc_f606w_jf5453c1_flc.fits to /Users/cwat/research/agel_hst/HST/AGEL051603-220847A/HST/17307_F606W/raw_data/mastDownload/HST/hst_17307_53_acs_wfc_f606w_jf5453c1/hst_17307_53_acs_wfc_f606w_jf5453c1_flc.fits ... [Done]\n",
      "Downloading URL https://mast.stsci.edu/api/v0.1/Download/file?uri=mast:HST/product/jf5453c0q_flc.fits to /Users/cwat/research/agel_hst/HST/AGEL051603-220847A/HST/17307_F606W/raw_data/mastDownload/HST/jf5453c0q/jf5453c0q_flc.fits ... [Done]\n",
      "Downloading URL https://mast.stsci.edu/api/v0.1/Download/file?uri=mast:HST/product/jf5453c1q_flc.fits to /Users/cwat/research/agel_hst/HST/AGEL051603-220847A/HST/17307_F606W/raw_data/mastDownload/HST/jf5453c1q/jf5453c1q_flc.fits ... [Done]\n"
     ]
    }
   ],
   "source": [
    "for i in Processed_Lenses:\n",
    "    if i[0][-1] != 'A':\n",
    "        continue  # skip B entries in Airtable for now (since they are duplicates from A entries)\n",
    "    for band in filters:\n",
    "        Downloader(i[0], \n",
    "                    i[1], \n",
    "                    band, \n",
    "                    camera, \n",
    "                    proposal_id_no)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7eae6127",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "provenance": []
  },
  "kernelspec": {
   "display_name": "stenv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
